<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
<meta name="theme-color" content="#222">
<meta name="generator" content="Hexo 4.0.0">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/logo.svg" color="#222">
  <link rel="alternate" href="/atom.xml" title="AiChery" type="application/atom+xml">

<link rel="stylesheet" href="/css/main.css">

<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic&display=swap&subset=latin,latin-ext">
<link rel="stylesheet" href="/lib/font-awesome/css/font-awesome.min.css">


<script id="hexo-configurations">
  var NexT = window.NexT || {};
  var CONFIG = {
    root: '/',
    scheme: 'Gemini',
    version: '7.5.0',
    exturl: false,
    sidebar: {"position":"left","display":"post","offset":12,"onmobile":false},
    copycode: {"enable":false,"show_result":false,"style":null},
    back2top: {"enable":true,"sidebar":false,"scrollpercent":false},
    bookmark: {"enable":false,"color":"#222","save":"auto"},
    fancybox: false,
    mediumzoom: false,
    lazyload: false,
    pangu: false,
    algolia: {
      appID: '',
      apiKey: '',
      indexName: '',
      hits: {"per_page":10},
      labels: {"input_placeholder":"Search for Posts","hits_empty":"We didn't find any results for the search: ${query}","hits_stats":"${hits} results found in ${time} ms"}
    },
    localsearch: {"enable":false,"trigger":"auto","top_n_per_article":1,"unescape":false,"preload":false},
    path: '',
    motion: {"enable":true,"async":false,"transition":{"post_block":"fadeIn","post_header":"slideDownIn","post_body":"slideDownIn","coll_header":"slideLeftIn","sidebar":"slideUpIn"}},
    translation: {
      copy_button: '复制',
      copy_success: '复制成功',
      copy_failure: '复制失败'
    },
    sidebarPadding: 40
  };
</script>

  <meta name="description" content="计算机视觉领域关于自注意力机制的一篇重要文章。">
<meta name="keywords" content="attention">
<meta property="og:type" content="article">
<meta property="og:title" content="Non-local Neural Networks笔记">
<meta property="og:url" content="http:&#x2F;&#x2F;yoursite.com&#x2F;2019&#x2F;11&#x2F;11&#x2F;Non-local%20Neural%20Networks%E7%AC%94%E8%AE%B0&#x2F;index.html">
<meta property="og:site_name" content="AiChery">
<meta property="og:description" content="计算机视觉领域关于自注意力机制的一篇重要文章。">
<meta property="og:locale" content="zh-CN">
<meta property="og:image" content="https:&#x2F;&#x2F;blog-1253296122.cos.ap-chengdu.myqcloud.com&#x2F;attention_05.png">
<meta property="og:image" content="https:&#x2F;&#x2F;blog-1253296122.cos.ap-chengdu.myqcloud.com&#x2F;attention_06.png">
<meta property="og:image" content="https:&#x2F;&#x2F;blog-1253296122.cos.ap-chengdu.myqcloud.com&#x2F;attention_07.png">
<meta property="og:updated_time" content="2019-11-11T06:18:40.926Z">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https:&#x2F;&#x2F;blog-1253296122.cos.ap-chengdu.myqcloud.com&#x2F;attention_05.png">

<link rel="canonical" href="http://yoursite.com/2019/11/11/Non-local%20Neural%20Networks%E7%AC%94%E8%AE%B0/">


<script id="page-configurations">
  // https://hexo.io/docs/variables.html
  CONFIG.page = {
    sidebar: "",
    isHome: false,
    isPost: true,
    isPage: false,
    isArchive: false
  };
</script>

  <title>Non-local Neural Networks笔记 | AiChery</title>
  






  <noscript>
  <style>
  .use-motion .brand,
  .use-motion .menu-item,
  .sidebar-inner,
  .use-motion .post-block,
  .use-motion .pagination,
  .use-motion .comments,
  .use-motion .post-header,
  .use-motion .post-body,
  .use-motion .collection-header { opacity: initial; }

  .use-motion .site-title,
  .use-motion .site-subtitle {
    opacity: initial;
    top: initial;
  }

  .use-motion .logo-line-before i { left: initial; }
  .use-motion .logo-line-after i { right: initial; }
  </style>
</noscript>

</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container use-motion">
    <div class="headband"></div>

    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner"><div class="site-brand-container">
  <div class="site-meta">

    <div>
      <a href="/" class="brand" rel="start">
        <span class="logo-line-before"><i></i></span>
        <span class="site-title">AiChery</span>
        <span class="logo-line-after"><i></i></span>
      </a>
    </div>
        <p class="site-subtitle">Life is about waiting for the right moment to act.</p>
  </div>

  <div class="site-nav-toggle">
    <div class="toggle" aria-label="切换导航栏">
      <span class="toggle-line toggle-line-first"></span>
      <span class="toggle-line toggle-line-middle"></span>
      <span class="toggle-line toggle-line-last"></span>
    </div>
  </div>
</div>


<nav class="site-nav">
  
  <ul id="menu" class="menu">
        <li class="menu-item menu-item-home">

    <a href="/" rel="section"><i class="fa fa-fw fa-home"></i>首页</a>

  </li>
        <li class="menu-item menu-item-about">

    <a href="/about/" rel="section"><i class="fa fa-fw fa-user"></i>关于</a>

  </li>
        <li class="menu-item menu-item-tags">

    <a href="/tags/" rel="section"><i class="fa fa-fw fa-tags"></i>标签</a>

  </li>
        <li class="menu-item menu-item-categories">

    <a href="/categories/" rel="section"><i class="fa fa-fw fa-th"></i>分类</a>

  </li>
        <li class="menu-item menu-item-archives">

    <a href="/archives/" rel="section"><i class="fa fa-fw fa-archive"></i>归档</a>

  </li>
        <li class="menu-item menu-item-简历">

    <a href="/Resume/" rel="section"><i class="fa fa-fw fa-th"></i>简历</a>

  </li>
  </ul>

</nav>
</div>
    </header>

    
  <div class="back-to-top">
    <i class="fa fa-arrow-up"></i>
    <span>0%</span>
  </div>


    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          

          <div class="content">
            

  <div class="posts-expand">
      
  
  
  <article itemscope itemtype="http://schema.org/Article" class="post-block " lang="zh-CN">
    <link itemprop="mainEntityOfPage" href="http://yoursite.com/2019/11/11/Non-local%20Neural%20Networks%E7%AC%94%E8%AE%B0/">

    <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
      <meta itemprop="image" content="https://blog-1253296122.cos.ap-chengdu.myqcloud.com/avatar.jpg">
      <meta itemprop="name" content="Wei Cheney">
      <meta itemprop="description" content="谦谦君子，温润如玉">
    </span>

    <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
      <meta itemprop="name" content="AiChery">
    </span>
      <header class="post-header">
        <h1 class="post-title" itemprop="name headline">
          Non-local Neural Networks笔记
        </h1>

        <div class="post-meta">
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-calendar-o"></i>
              </span>
              <span class="post-meta-item-text">发表于</span>
              

              <time title="创建时间：2019-11-11 10:19:11 / 修改时间：14:18:40" itemprop="dateCreated datePublished" datetime="2019-11-11T10:19:11+08:00">2019-11-11</time>
            </span>
            <span class="post-meta-item">
              <span class="post-meta-item-icon">
                <i class="fa fa-folder-o"></i>
              </span>
              <span class="post-meta-item-text">分类于</span>
                <span itemprop="about" itemscope itemtype="http://schema.org/Thing">
                  <a href="/categories/%E8%AE%A1%E7%AE%97%E6%9C%BA%E8%A7%86%E8%A7%89/" itemprop="url" rel="index">
                    <span itemprop="name">计算机视觉</span>
                  </a>
                </span>
            </span>

          
            <span class="post-meta-item" title="阅读次数" id="busuanzi_container_page_pv" style="display: none;">
              <span class="post-meta-item-icon">
                <i class="fa fa-eye"></i>
              </span>
              <span class="post-meta-item-text">阅读次数：</span>
              <span id="busuanzi_value_page_pv"></span>
            </span>

        </div>
      </header>

    
    
    
    <div class="post-body" itemprop="articleBody">

      
        <p>计算机视觉领域关于自注意力机制的一篇重要文章。</p>
<a id="more"></a>
<p>CNN和 RNN 每次计算只能处理一个 local neighborhood，Non-local Neural Networks 提出了 non-local operations 的想法，以捕获  long-range dependencies. Non-local operation 在计算某个位置的 response 时，考虑所有位置，并进行加和权重。次方法解决了解決即使相距很远的的 blocks，仍然有可能是彼此有关系的。</p>
<h2 id="1-Introduction"><a href="#1-Introduction" class="headerlink" title="1. Introduction"></a>1. Introduction</h2><p>CNN和 RNN 每次计算只能处理一个 local neighborhood, 想要捕捉 long-range dependencies，必须使用很深的网络，造成：</p>
<ul>
<li>计算不够高效</li>
<li>难以优化</li>
<li>non-local 特征讯息传递不灵活</li>
</ul>
<div class="table-container">
<table>
<thead>
<tr>
<th style="text-align:center"><img src="https://blog-1253296122.cos.ap-chengdu.myqcloud.com/attention_05.png" alt="img"></th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:center">Figure 1. A spacetime <strong><em>non-local</em></strong> operation in our network trained for video classification in Kinetics. A position $x_i$’s response is computed by the weighted average of the features of <em>all</em> positions $x_j$ (only the highest weighted ones are shown here). In this example computed by our model, note how it relates the ball in the first frame to the ball in the last two frames. More examples are in Figure 3.</td>
</tr>
</tbody>
</table>
</div>
<p>如上图所示，本文提出了 <em>non-local</em> operations ，一个 non-local 的值 $x_i $ 来自作为输入 input feature maps 中所有位置 $x_j ​$ 的加权和。</p>
<p>使用 non-local operations 的几个优点：</p>
<ul>
<li>通过计算任意两个位置间的interactions，non-local operations 可以直接捕获其  long-range dependencies，无关它们之间的距离。</li>
<li>非常高效，即使只有几层也能取得很好的效果。</li>
<li>non-local operations 保持了输入大小，可以很容易地结合到其它网络中。</li>
</ul>
<h2 id="2-Related-Work"><a href="#2-Related-Work" class="headerlink" title="2. Related Work"></a>2. Related Work</h2><p><strong>Non-local image processing. </strong> 来源于 Non-local means。</p>
<p><strong>Graphical models.</strong> 可以通过图模型，如CRF，来捕获 long-range dependencies，CRF可以作为后处理步骤，也可以融合到RNN中。但是本文的方法更加简单，也与图神经网络有关系。</p>
<p><strong>Feedforward modeling for sequences.</strong> Recently there emerged a trend of using feedforward (i.e., non-recurrent) networks for modeling sequences in speech and language. In these methods, long-term dependencies are captured by the large receptive fields contributed by very deep 1-D convolutions. These feedforward models are amenable to parallelized implementations and can be more<br>efficient than widely used recurrent models.</p>
<p><strong>Self-attention.</strong> 本文方法与 <a href="https://arxiv.org/abs/1706.03762" target="_blank" rel="noopener">self-attention</a> 方法有关，self-attention 模块通过考虑所有位置并在 embedding space 中获取它们的加权平均值，来计算序列（例如句子）中某个位置的响应。作者在后面有讨论， self-attention 可以视为某种形式的 <a href="https://ieeexplore.ieee.org/document/1467423" target="_blank" rel="noopener">non-local mean</a>, 从这角度来说，将机器翻译中的  self-attention 机制扩展到 CV 领域。</p>
<p><strong>Interaction networks.</strong> </p>
<p><strong>Video classification architectures. </strong></p>
<h2 id="3-Non-local-Neural-Networks"><a href="#3-Non-local-Neural-Networks" class="headerlink" title="3. Non-local Neural Networks"></a>3. Non-local Neural Networks</h2><h3 id="3-1-Formulation"><a href="#3-1-Formulation" class="headerlink" title="3.1. Formulation"></a>3.1. Formulation</h3><p>通过<a href="https://zh.wikipedia.org/wiki/%E9%9D%9E%E5%B1%80%E9%83%A8%E5%B9%B3%E5%9D%87" target="_blank" rel="noopener">非局部平均</a>，作者定义了如下的 non-local operation：</p>
<script type="math/tex; mode=display">
\textbf y_i = \frac{1}{\mathcal C(\textbf x)} \sum_{\forall j} f(\textbf x_i, \textbf x_j) g(\textbf x_j). \tag{1}</script><p>其中，</p>
<ul>
<li>$i​$：输出位置的 index（in space, time, or spacetime）</li>
<li>$j​$：所有 enumerates 出来可能位置的 index</li>
<li>$\textbf x​$:  输入（image, sequence, video; often their features）</li>
<li>$\textbf y​$:  输出，大小和 $\textbf x​$ 相同</li>
<li>$f​$：计算 $i​$ 和所有 $j​$ 之间的 affinity</li>
<li>$g​$：计算位置 $j​$ 处的输入信号表示，a representation of the input signal at the position $j​$</li>
<li>$\mathcal C​$：normalizer</li>
</ul>
<p>可以清楚看出 non-local 的特性，考虑了所有的位置 $\forall j​$. 相比 CNN 操作只关心邻域和 RNN 只关心前一个时间点。此外，non-local operation 和 fully-connected  相比有许多优点：</p>
<div class="table-container">
<table>
<thead>
<tr>
<th style="text-align:center">non-local operation</th>
<th style="text-align:center">fc</th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:center">考虑不同位置的关系</td>
<td style="text-align:center">单纯学习权重</td>
</tr>
<tr>
<td style="text-align:center">支持不同的 input size</td>
<td style="text-align:center">固定大小</td>
</tr>
<tr>
<td style="text-align:center">容易融合到各种网络中</td>
<td style="text-align:center">只能接在网络最后面</td>
</tr>
</tbody>
</table>
</div>
<h3 id="3-2-Instantiations"><a href="#3-2-Instantiations" class="headerlink" title="3.2. Instantiations"></a>3.2. Instantiations</h3><p>接下来，描述不同版本的 $f$ 和 $g​$，作者发现，这些选择不太影响 models，表示了：non-local 想法才是改善的主因。</p>
<p>为了方便描述，我们只考虑 $g$ 是一个 linear embedding：$g(\textbf x_j) = W_g \textbf x_j$，其中 $W_g$ 是一个 weight matrix，下面介绍不同的 $f$（基本上，就是找出位置 $i$ 和所有位置 $j$ 的关系），和他们对应的 normalizer $\mathcal C$。</p>
<h4 id="Gaussian"><a href="#Gaussian" class="headerlink" title="Gaussian"></a>Gaussian</h4><script type="math/tex; mode=display">
f(\textbf x_i, \textbf x_j) = e^{\textbf x_i^T \textbf x_j}. \tag{2}</script><p>其中：</p>
<ul>
<li>$\textbf x_i^T \textbf x_j$: 点积相似度（欧式距离也 ok，但点积较好实作）</li>
<li>$\mathcal C(\textbf x) = \sum_{\forall j} f(\textbf x_i, \textbf x_j)$</li>
</ul>
<h4 id="Embedded-Gaussian"><a href="#Embedded-Gaussian" class="headerlink" title="Embedded Gaussian"></a>Embedded Gaussian</h4><script type="math/tex; mode=display">
f(\textbf x_i, \textbf x_j) = e^{\theta(\textbf x_i)^T \phi(\textbf x_j)}. \tag{3}</script><p>其中：</p>
<ul>
<li>$\theta(\textbf x_i) = W_\theta \textbf x_i​$</li>
<li>$\phi(\textbf x_j) = W_\phi \textbf x_j$</li>
<li>$\mathcal C(\textbf x) = \sum_{\forall j} f(\textbf x_i, \textbf x_j)$</li>
</ul>
<p>作者指出，<a href="https://arxiv.org/abs/1706.03762" target="_blank" rel="noopener">self-attention module</a> 是这个版本的 non-local operations 的一个特例。可以观察到，给定一个 $i$, $\frac{1}{\mathcal C(x)} f(\textbf x_i, \textbf x_j)$ 是透过维度 $j$ 的 <a href="https://zh.wikipedia.org/wiki/Softmax%E5%87%BD%E6%95%B0" target="_blank" rel="noopener"><em>softmax</em></a>. 所以，我们有 $\textbf y =softmax(\textbf x^T W_{\theta}^T W_{\phi} \textbf x)g(\textbf x) ​$。 </p>
<h4 id="Dot-product"><a href="#Dot-product" class="headerlink" title="Dot product"></a>Dot product</h4><script type="math/tex; mode=display">
f(\textbf x_i, \textbf x_j) = \theta(\textbf x_i)^T \phi(\textbf x_j). \tag{4}</script><p>其中：</p>
<ul>
<li>$\theta(\textbf x_i) = W_\theta \textbf x_i$</li>
<li>$\phi(\textbf x_j) = W_\phi \textbf x_j$</li>
<li>$\mathcal C(\textbf x) = N$</li>
</ul>
<h4 id="Concatenation"><a href="#Concatenation" class="headerlink" title="Concatenation"></a>Concatenation</h4><script type="math/tex; mode=display">
f(\textbf x_i, \textbf x_j) = \text{ReLU} (\textbf w_f^T[\theta(\textbf x_i), \phi(\textbf x_j)]). \tag{5}</script><p>其中：</p>
<ul>
<li>$[\cdot, \cdot]$: concatenation\</li>
<li>$\textbf w_f$: 将 concatenated vector 投射成 scalar 的 weight vector</li>
<li>$\mathcal C(\textbf x) = N$: </li>
<li>在这里 $f$ 多采用了一个 ReLU</li>
</ul>
<h3 id="3-3-Non-local-Block"><a href="#3-3-Non-local-Block" class="headerlink" title="3.3. Non-local Block"></a>3.3. Non-local Block</h3><p>我们将式 (1) 中的 non-local operation 包装到 non-local block 中，该 block 可以合并到许多现有体系的架构中，non-local block 定义如下：</p>
<script type="math/tex; mode=display">
\textbf z_i = W_z \textbf y_i + \textbf x_i, \tag{6}</script><p>应用了残差结构，可以在任何事先训练好的 model 插入一个 non-local block。</p>
<div class="table-container">
<table>
<thead>
<tr>
<th style="text-align:center"><img src="https://blog-1253296122.cos.ap-chengdu.myqcloud.com/attention_06.png" alt="img"></th>
</tr>
</thead>
<tbody>
<tr>
<td style="text-align:center">Figure 2. A spacetime <strong>non-local block</strong>. The feature maps are shown as the shape of their tensors, e.g., $T \times H \times W \times 1024$ for $1024$ channels (proper reshaping is performed when noted). “$\otimes$” denotes matrix multiplication, and “$\oplus$” denotes element-wise sum. The softmax operation is performed on each row. The blue boxes denote $1 \times 1 \times 1$ convolutions. Here we show the embedded Gaussian version, with a bottleneck of $512$ channels. The vanilla Gaussian version can be done by removing $\theta$ and $\phi$, and the dot-product version can be done by replacing softmax with scaling by $1 / N$.</td>
</tr>
</tbody>
</table>
</div>
<p> <a href="https://walkccc.github.io/blog/2018/10/27/Papers/nonlocal-nn/" target="_blank" rel="noopener">博客</a> 画了一个示意图：</p>
<p><img src="https://blog-1253296122.cos.ap-chengdu.myqcloud.com/attention_07.png" alt=""></p>
<h2 id="4-代码实现"><a href="#4-代码实现" class="headerlink" title="4. 代码实现"></a>4. 代码实现</h2><p>参考：<a href="https://github.com/AlexHex7/Non-local_pytorch" target="_blank" rel="noopener">https://github.com/AlexHex7/Non-local_pytorch</a></p>
<p>基于PyTorch</p>
<h4 id="Gaussian-1"><a href="#Gaussian-1" class="headerlink" title="Gaussian"></a>Gaussian</h4><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">_NonLocalBlockND</span><span class="params">(nn.Module)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, in_channels, inter_channels=None, dimension=<span class="number">3</span>, sub_sample=True, bn_layer=True)</span>:</span></span><br><span class="line">        super(_NonLocalBlockND, self).__init__()</span><br><span class="line"></span><br><span class="line">        <span class="keyword">assert</span> dimension <span class="keyword">in</span> [<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]</span><br><span class="line"></span><br><span class="line">        self.dimension = dimension</span><br><span class="line">        self.sub_sample = sub_sample</span><br><span class="line"></span><br><span class="line">        self.in_channels = in_channels</span><br><span class="line">        self.inter_channels = inter_channels</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> self.inter_channels <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            self.inter_channels = in_channels // <span class="number">2</span></span><br><span class="line">            <span class="keyword">if</span> self.inter_channels == <span class="number">0</span>:</span><br><span class="line">                self.inter_channels = <span class="number">1</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> dimension == <span class="number">3</span>:</span><br><span class="line">            conv_nd = nn.Conv3d</span><br><span class="line">            max_pool_layer = nn.MaxPool3d(kernel_size=(<span class="number">1</span>, <span class="number">2</span>, <span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm3d</span><br><span class="line">        <span class="keyword">elif</span> dimension == <span class="number">2</span>:</span><br><span class="line">            conv_nd = nn.Conv2d</span><br><span class="line">            max_pool_layer = nn.MaxPool2d(kernel_size=(<span class="number">2</span>, <span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm2d</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            conv_nd = nn.Conv1d</span><br><span class="line">            max_pool_layer = nn.MaxPool1d(kernel_size=(<span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm1d</span><br><span class="line"></span><br><span class="line">        self.g = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                         kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> bn_layer:</span><br><span class="line">            self.W = nn.Sequential(</span><br><span class="line">                conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,</span><br><span class="line">                        kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>),</span><br><span class="line">                bn(self.in_channels)</span><br><span class="line">            )</span><br><span class="line">            nn.init.constant_(self.W[<span class="number">1</span>].weight, <span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W[<span class="number">1</span>].bias, <span class="number">0</span>)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            self.W = conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,</span><br><span class="line">                             kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W.weight, <span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W.bias, <span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> sub_sample:</span><br><span class="line">            self.g = nn.Sequential(self.g, max_pool_layer)</span><br><span class="line">            self.phi = max_pool_layer</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">forward</span><span class="params">(self, x)</span>:</span></span><br><span class="line">        <span class="string">'''</span></span><br><span class="line"><span class="string">        :param x: (b, c, t, h, w)</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">        '''</span></span><br><span class="line"></span><br><span class="line">        batch_size = x.size(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        g_x = self.g(x).view(batch_size, self.inter_channels, <span class="number">-1</span>) <span class="comment"># 将宽和高展平</span></span><br><span class="line"></span><br><span class="line">        g_x = g_x.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>) <span class="comment"># batchsize * HW * Channel</span></span><br><span class="line"></span><br><span class="line">        theta_x = x.view(batch_size, self.in_channels, <span class="number">-1</span>)</span><br><span class="line">        theta_x = theta_x.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> self.sub_sample:</span><br><span class="line">            phi_x = self.phi(x).view(batch_size, self.in_channels, <span class="number">-1</span>)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            phi_x = x.view(batch_size, self.in_channels, <span class="number">-1</span>)</span><br><span class="line"></span><br><span class="line">        f = torch.matmul(theta_x, phi_x)</span><br><span class="line">        f_div_C = F.softmax(f, dim=<span class="number">-1</span>)</span><br><span class="line"></span><br><span class="line">        y = torch.matmul(f_div_C, g_x)</span><br><span class="line">        y = y.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>).contiguous()</span><br><span class="line">        y = y.view(batch_size, self.inter_channels, *x.size()[<span class="number">2</span>:])</span><br><span class="line">        W_y = self.W(y)</span><br><span class="line">        z = W_y + x</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> z</span><br></pre></td></tr></table></figure>
<h4 id="Embedded-Gaussian-1"><a href="#Embedded-Gaussian-1" class="headerlink" title="Embedded Gaussian"></a>Embedded Gaussian</h4><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">_NonLocalBlockND</span><span class="params">(nn.Module)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, in_channels, inter_channels=None, dimension=<span class="number">3</span>, sub_sample=True, bn_layer=True)</span>:</span></span><br><span class="line">        super(_NonLocalBlockND, self).__init__()</span><br><span class="line"></span><br><span class="line">        <span class="keyword">assert</span> dimension <span class="keyword">in</span> [<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]</span><br><span class="line"></span><br><span class="line">        self.dimension = dimension</span><br><span class="line">        self.sub_sample = sub_sample</span><br><span class="line"></span><br><span class="line">        self.in_channels = in_channels</span><br><span class="line">        self.inter_channels = inter_channels</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> self.inter_channels <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            self.inter_channels = in_channels // <span class="number">2</span></span><br><span class="line">            <span class="keyword">if</span> self.inter_channels == <span class="number">0</span>:</span><br><span class="line">                self.inter_channels = <span class="number">1</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> dimension == <span class="number">3</span>:</span><br><span class="line">            conv_nd = nn.Conv3d</span><br><span class="line">            max_pool_layer = nn.MaxPool3d(kernel_size=(<span class="number">1</span>, <span class="number">2</span>, <span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm3d</span><br><span class="line">        <span class="keyword">elif</span> dimension == <span class="number">2</span>:</span><br><span class="line">            conv_nd = nn.Conv2d</span><br><span class="line">            max_pool_layer = nn.MaxPool2d(kernel_size=(<span class="number">2</span>, <span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm2d</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            conv_nd = nn.Conv1d</span><br><span class="line">            max_pool_layer = nn.MaxPool1d(kernel_size=(<span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm1d</span><br><span class="line"></span><br><span class="line">        self.g = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                         kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> bn_layer:</span><br><span class="line">            self.W = nn.Sequential(</span><br><span class="line">                conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,</span><br><span class="line">                        kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>),</span><br><span class="line">                bn(self.in_channels)</span><br><span class="line">            )</span><br><span class="line">            nn.init.constant_(self.W[<span class="number">1</span>].weight, <span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W[<span class="number">1</span>].bias, <span class="number">0</span>)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            self.W = conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,</span><br><span class="line">                             kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W.weight, <span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W.bias, <span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        self.theta = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                             kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line">        self.phi = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                           kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> sub_sample:</span><br><span class="line">            self.g = nn.Sequential(self.g, max_pool_layer)</span><br><span class="line">            self.phi = nn.Sequential(self.phi, max_pool_layer)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">forward</span><span class="params">(self, x)</span>:</span></span><br><span class="line">        <span class="string">'''</span></span><br><span class="line"><span class="string">        :param x: (b, c, t, h, w)</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">        '''</span></span><br><span class="line"></span><br><span class="line">        batch_size = x.size(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        g_x = self.g(x).view(batch_size, self.inter_channels, <span class="number">-1</span>)</span><br><span class="line">        g_x = g_x.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        theta_x = self.theta(x).view(batch_size, self.inter_channels, <span class="number">-1</span>)</span><br><span class="line">        theta_x = theta_x.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>)</span><br><span class="line">        phi_x = self.phi(x).view(batch_size, self.inter_channels, <span class="number">-1</span>)</span><br><span class="line">        f = torch.matmul(theta_x, phi_x)</span><br><span class="line">        f_div_C = F.softmax(f, dim=<span class="number">-1</span>)</span><br><span class="line"></span><br><span class="line">        y = torch.matmul(f_div_C, g_x)</span><br><span class="line">        y = y.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>).contiguous()</span><br><span class="line">        y = y.view(batch_size, self.inter_channels, *x.size()[<span class="number">2</span>:])</span><br><span class="line">        W_y = self.W(y)</span><br><span class="line">        z = W_y + x</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> z</span><br></pre></td></tr></table></figure>
<h4 id="Dot-product-1"><a href="#Dot-product-1" class="headerlink" title="Dot product"></a>Dot product</h4><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">_NonLocalBlockND</span><span class="params">(nn.Module)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, in_channels, inter_channels=None, dimension=<span class="number">3</span>, sub_sample=True, bn_layer=True)</span>:</span></span><br><span class="line">        super(_NonLocalBlockND, self).__init__()</span><br><span class="line"></span><br><span class="line">        <span class="keyword">assert</span> dimension <span class="keyword">in</span> [<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]</span><br><span class="line"></span><br><span class="line">        self.dimension = dimension</span><br><span class="line">        self.sub_sample = sub_sample</span><br><span class="line"></span><br><span class="line">        self.in_channels = in_channels</span><br><span class="line">        self.inter_channels = inter_channels</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> self.inter_channels <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            self.inter_channels = in_channels // <span class="number">2</span></span><br><span class="line">            <span class="keyword">if</span> self.inter_channels == <span class="number">0</span>:</span><br><span class="line">                self.inter_channels = <span class="number">1</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> dimension == <span class="number">3</span>:</span><br><span class="line">            conv_nd = nn.Conv3d</span><br><span class="line">            max_pool_layer = nn.MaxPool3d(kernel_size=(<span class="number">1</span>, <span class="number">2</span>, <span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm3d</span><br><span class="line">        <span class="keyword">elif</span> dimension == <span class="number">2</span>:</span><br><span class="line">            conv_nd = nn.Conv2d</span><br><span class="line">            max_pool_layer = nn.MaxPool2d(kernel_size=(<span class="number">2</span>, <span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm2d</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            conv_nd = nn.Conv1d</span><br><span class="line">            max_pool_layer = nn.MaxPool1d(kernel_size=(<span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm1d</span><br><span class="line"></span><br><span class="line">        self.g = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                         kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> bn_layer:</span><br><span class="line">            self.W = nn.Sequential(</span><br><span class="line">                conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,</span><br><span class="line">                        kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>),</span><br><span class="line">                bn(self.in_channels)</span><br><span class="line">            )</span><br><span class="line">            nn.init.constant_(self.W[<span class="number">1</span>].weight, <span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W[<span class="number">1</span>].bias, <span class="number">0</span>)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            self.W = conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,</span><br><span class="line">                             kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W.weight, <span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W.bias, <span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        self.theta = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                             kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        self.phi = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                           kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> sub_sample:</span><br><span class="line">            self.g = nn.Sequential(self.g, max_pool_layer)</span><br><span class="line">            self.phi = nn.Sequential(self.phi, max_pool_layer)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">forward</span><span class="params">(self, x)</span>:</span></span><br><span class="line">        <span class="string">'''</span></span><br><span class="line"><span class="string">        :param x: (b, c, t, h, w)</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">        '''</span></span><br><span class="line"></span><br><span class="line">        batch_size = x.size(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        g_x = self.g(x).view(batch_size, self.inter_channels, <span class="number">-1</span>)</span><br><span class="line">        g_x = g_x.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        theta_x = self.theta(x).view(batch_size, self.inter_channels, <span class="number">-1</span>)</span><br><span class="line">        theta_x = theta_x.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>)</span><br><span class="line">        phi_x = self.phi(x).view(batch_size, self.inter_channels, <span class="number">-1</span>)</span><br><span class="line">        f = torch.matmul(theta_x, phi_x)</span><br><span class="line">        N = f.size(<span class="number">-1</span>)</span><br><span class="line">        f_div_C = f / N</span><br><span class="line"></span><br><span class="line">        y = torch.matmul(f_div_C, g_x)</span><br><span class="line">        y = y.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>).contiguous()</span><br><span class="line">        y = y.view(batch_size, self.inter_channels, *x.size()[<span class="number">2</span>:])</span><br><span class="line">        W_y = self.W(y)</span><br><span class="line">        z = W_y + x</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> z</span><br></pre></td></tr></table></figure>
<h4 id="Concatenation-1"><a href="#Concatenation-1" class="headerlink" title="Concatenation"></a>Concatenation</h4><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br><span class="line">79</span><br><span class="line">80</span><br><span class="line">81</span><br><span class="line">82</span><br><span class="line">83</span><br><span class="line">84</span><br><span class="line">85</span><br><span class="line">86</span><br><span class="line">87</span><br><span class="line">88</span><br><span class="line">89</span><br><span class="line">90</span><br><span class="line">91</span><br><span class="line">92</span><br><span class="line">93</span><br><span class="line">94</span><br><span class="line">95</span><br><span class="line">96</span><br><span class="line">97</span><br><span class="line">98</span><br></pre></td><td class="code"><pre><span class="line"><span class="class"><span class="keyword">class</span> <span class="title">_NonLocalBlockND</span><span class="params">(nn.Module)</span>:</span></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">__init__</span><span class="params">(self, in_channels, inter_channels=None, dimension=<span class="number">3</span>, sub_sample=True, bn_layer=True)</span>:</span></span><br><span class="line">        super(_NonLocalBlockND, self).__init__()</span><br><span class="line"></span><br><span class="line">        <span class="keyword">assert</span> dimension <span class="keyword">in</span> [<span class="number">1</span>, <span class="number">2</span>, <span class="number">3</span>]</span><br><span class="line"></span><br><span class="line">        self.dimension = dimension</span><br><span class="line">        self.sub_sample = sub_sample</span><br><span class="line"></span><br><span class="line">        self.in_channels = in_channels</span><br><span class="line">        self.inter_channels = inter_channels</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> self.inter_channels <span class="keyword">is</span> <span class="literal">None</span>:</span><br><span class="line">            self.inter_channels = in_channels // <span class="number">2</span></span><br><span class="line">            <span class="keyword">if</span> self.inter_channels == <span class="number">0</span>:</span><br><span class="line">                self.inter_channels = <span class="number">1</span></span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> dimension == <span class="number">3</span>:</span><br><span class="line">            conv_nd = nn.Conv3d</span><br><span class="line">            max_pool_layer = nn.MaxPool3d(kernel_size=(<span class="number">1</span>, <span class="number">2</span>, <span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm3d</span><br><span class="line">        <span class="keyword">elif</span> dimension == <span class="number">2</span>:</span><br><span class="line">            conv_nd = nn.Conv2d</span><br><span class="line">            max_pool_layer = nn.MaxPool2d(kernel_size=(<span class="number">2</span>, <span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm2d</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            conv_nd = nn.Conv1d</span><br><span class="line">            max_pool_layer = nn.MaxPool1d(kernel_size=(<span class="number">2</span>))</span><br><span class="line">            bn = nn.BatchNorm1d</span><br><span class="line"></span><br><span class="line">        self.g = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                         kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> bn_layer:</span><br><span class="line">            self.W = nn.Sequential(</span><br><span class="line">                conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,</span><br><span class="line">                        kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>),</span><br><span class="line">                bn(self.in_channels)</span><br><span class="line">            )</span><br><span class="line">            nn.init.constant_(self.W[<span class="number">1</span>].weight, <span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W[<span class="number">1</span>].bias, <span class="number">0</span>)</span><br><span class="line">        <span class="keyword">else</span>:</span><br><span class="line">            self.W = conv_nd(in_channels=self.inter_channels, out_channels=self.in_channels,</span><br><span class="line">                             kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W.weight, <span class="number">0</span>)</span><br><span class="line">            nn.init.constant_(self.W.bias, <span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        self.theta = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                             kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        self.phi = conv_nd(in_channels=self.in_channels, out_channels=self.inter_channels,</span><br><span class="line">                           kernel_size=<span class="number">1</span>, stride=<span class="number">1</span>, padding=<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        self.concat_project = nn.Sequential(</span><br><span class="line">            nn.Conv2d(self.inter_channels * <span class="number">2</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>, <span class="number">0</span>, bias=<span class="literal">False</span>),</span><br><span class="line">            nn.ReLU()</span><br><span class="line">        )</span><br><span class="line"></span><br><span class="line">        <span class="keyword">if</span> sub_sample:</span><br><span class="line">            self.g = nn.Sequential(self.g, max_pool_layer)</span><br><span class="line">            self.phi = nn.Sequential(self.phi, max_pool_layer)</span><br><span class="line"></span><br><span class="line">    <span class="function"><span class="keyword">def</span> <span class="title">forward</span><span class="params">(self, x)</span>:</span></span><br><span class="line">        <span class="string">'''</span></span><br><span class="line"><span class="string">        :param x: (b, c, t, h, w)</span></span><br><span class="line"><span class="string">        :return:</span></span><br><span class="line"><span class="string">        '''</span></span><br><span class="line"></span><br><span class="line">        batch_size = x.size(<span class="number">0</span>)</span><br><span class="line"></span><br><span class="line">        g_x = self.g(x).view(batch_size, self.inter_channels, <span class="number">-1</span>)</span><br><span class="line">        g_x = g_x.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        <span class="comment"># (b, c, N, 1)</span></span><br><span class="line">        theta_x = self.theta(x).view(batch_size, self.inter_channels, <span class="number">-1</span>, <span class="number">1</span>)</span><br><span class="line">        <span class="comment"># (b, c, 1, N)</span></span><br><span class="line">        phi_x = self.phi(x).view(batch_size, self.inter_channels, <span class="number">1</span>, <span class="number">-1</span>)</span><br><span class="line"></span><br><span class="line">        h = theta_x.size(<span class="number">2</span>)</span><br><span class="line">        w = phi_x.size(<span class="number">3</span>)</span><br><span class="line">        theta_x = theta_x.repeat(<span class="number">1</span>, <span class="number">1</span>, <span class="number">1</span>, w)</span><br><span class="line">        phi_x = phi_x.repeat(<span class="number">1</span>, <span class="number">1</span>, h, <span class="number">1</span>)</span><br><span class="line"></span><br><span class="line">        concat_feature = torch.cat([theta_x, phi_x], dim=<span class="number">1</span>)</span><br><span class="line">        f = self.concat_project(concat_feature)</span><br><span class="line">        b, _, h, w = f.size()</span><br><span class="line">        f = f.view(b, h, w)</span><br><span class="line"></span><br><span class="line">        N = f.size(<span class="number">-1</span>)</span><br><span class="line">        f_div_C = f / N</span><br><span class="line"></span><br><span class="line">        y = torch.matmul(f_div_C, g_x)</span><br><span class="line">        y = y.permute(<span class="number">0</span>, <span class="number">2</span>, <span class="number">1</span>).contiguous()</span><br><span class="line">        y = y.view(batch_size, self.inter_channels, *x.size()[<span class="number">2</span>:])</span><br><span class="line">        W_y = self.W(y)</span><br><span class="line">        z = W_y + x</span><br><span class="line"></span><br><span class="line">        <span class="keyword">return</span> z</span><br></pre></td></tr></table></figure>
<p>参考资料:</p>
<p>[1] <a href="https://arxiv.org/abs/1711.07971" target="_blank" rel="noopener">Non-local Neural Networks</a></p>
<p>[2] <a href="https://walkccc.github.io/blog/2018/10/27/Papers/nonlocal-nn/" target="_blank" rel="noopener">論文筆記 Non-local Neural Networks</a></p>

    </div>

    
    
    
      
        <div class="reward-container">
  <div></div>
  <button disable="enable" onclick="var qr = document.getElementById(&quot;qr&quot;); qr.style.display = (qr.style.display === 'none') ? 'block' : 'none';">
    打赏
  </button>
  <div id="qr" style="display: none;">
      
      <div style="display: inline-block;">
        <img src="/images/wechatpay.png" alt="Wei Cheney 微信支付">
        <p>微信支付</p>
      </div>
      
      <div style="display: inline-block;">
        <img src="/images/alipay.png" alt="Wei Cheney 支付宝">
        <p>支付宝</p>
      </div>

  </div>
</div>


      <footer class="post-footer">
          <div class="post-tags">
              <a href="/tags/attention/" rel="tag"># attention</a>
          </div>

        

          <div class="post-nav">
            <div class="post-nav-next post-nav-item">
                <a href="/2019/11/08/%E8%AE%A1%E7%AE%97%E6%9C%BA%E8%A7%86%E8%A7%89%E4%B8%AD%E7%9A%84Attention%E6%9C%BA%E5%88%B6/" rel="next" title="计算机视觉中的Attention机制">
                  <i class="fa fa-chevron-left"></i> 计算机视觉中的Attention机制
                </a>
            </div>

            <span class="post-nav-divider"></span>

            <div class="post-nav-prev post-nav-item">
                <a href="/2019/11/11/CBAM%20Convolutional%20Block%20Attention%20Module%E7%AC%94%E8%AE%B0/" rel="prev" title="CBAM Convolutional Block Attention Module笔记">
                  CBAM Convolutional Block Attention Module笔记 <i class="fa fa-chevron-right"></i>
                </a>
            </div>
          </div>
      </footer>
    
  </article>
  
  
  

  </div>


          </div>
          
    <div class="comments" id="gitalk-container"></div>

        </div>
          
  
  <div class="toggle sidebar-toggle">
    <span class="toggle-line toggle-line-first"></span>
    <span class="toggle-line toggle-line-middle"></span>
    <span class="toggle-line toggle-line-last"></span>
  </div>

  <aside class="sidebar">
    <div class="sidebar-inner">

      <ul class="sidebar-nav motion-element">
        <li class="sidebar-nav-toc">
          文章目录
        </li>
        <li class="sidebar-nav-overview">
          站点概览
        </li>
      </ul>

      <!--noindex-->
      <div class="post-toc-wrap sidebar-panel">
          <div class="post-toc motion-element"><ol class="nav"><li class="nav-item nav-level-2"><a class="nav-link" href="#1-Introduction"><span class="nav-number">1.</span> <span class="nav-text">1. Introduction</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#2-Related-Work"><span class="nav-number">2.</span> <span class="nav-text">2. Related Work</span></a></li><li class="nav-item nav-level-2"><a class="nav-link" href="#3-Non-local-Neural-Networks"><span class="nav-number">3.</span> <span class="nav-text">3. Non-local Neural Networks</span></a><ol class="nav-child"><li class="nav-item nav-level-3"><a class="nav-link" href="#3-1-Formulation"><span class="nav-number">3.1.</span> <span class="nav-text">3.1. Formulation</span></a></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-2-Instantiations"><span class="nav-number">3.2.</span> <span class="nav-text">3.2. Instantiations</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Gaussian"><span class="nav-number">3.2.1.</span> <span class="nav-text">Gaussian</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Embedded-Gaussian"><span class="nav-number">3.2.2.</span> <span class="nav-text">Embedded Gaussian</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Dot-product"><span class="nav-number">3.2.3.</span> <span class="nav-text">Dot product</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Concatenation"><span class="nav-number">3.2.4.</span> <span class="nav-text">Concatenation</span></a></li></ol></li><li class="nav-item nav-level-3"><a class="nav-link" href="#3-3-Non-local-Block"><span class="nav-number">3.3.</span> <span class="nav-text">3.3. Non-local Block</span></a></li></ol></li><li class="nav-item nav-level-2"><a class="nav-link" href="#4-代码实现"><span class="nav-number">4.</span> <span class="nav-text">4. 代码实现</span></a><ol class="nav-child"><li class="nav-item nav-level-4"><a class="nav-link" href="#Gaussian-1"><span class="nav-number">4.0.1.</span> <span class="nav-text">Gaussian</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Embedded-Gaussian-1"><span class="nav-number">4.0.2.</span> <span class="nav-text">Embedded Gaussian</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Dot-product-1"><span class="nav-number">4.0.3.</span> <span class="nav-text">Dot product</span></a></li><li class="nav-item nav-level-4"><a class="nav-link" href="#Concatenation-1"><span class="nav-number">4.0.4.</span> <span class="nav-text">Concatenation</span></a></li></ol></li></ol></li></ol></div>
      </div>
      <!--/noindex-->

      <div class="site-overview-wrap sidebar-panel">
        <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
  <img class="site-author-image" itemprop="image" alt="Wei Cheney"
    src="https://blog-1253296122.cos.ap-chengdu.myqcloud.com/avatar.jpg">
  <p class="site-author-name" itemprop="name">Wei Cheney</p>
  <div class="site-description" itemprop="description">谦谦君子，温润如玉</div>
</div>
<div class="site-state-wrap motion-element">
  <nav class="site-state">
      <div class="site-state-item site-state-posts">
          <a href="/archives/">
        
          <span class="site-state-item-count">17</span>
          <span class="site-state-item-name">日志</span>
        </a>
      </div>
      <div class="site-state-item site-state-categories">
            <a href="/categories/">
          
        <span class="site-state-item-count">3</span>
        <span class="site-state-item-name">分类</span></a>
      </div>
      <div class="site-state-item site-state-tags">
            <a href="/tags/">
          
        <span class="site-state-item-count">8</span>
        <span class="site-state-item-name">标签</span></a>
      </div>
  </nav>
</div>
  <div class="feed-link motion-element">
    <a href="/atom.xml" rel="alternate">
      <i class="fa fa-rss"></i>RSS
    </a>
  </div>
  <div class="links-of-author motion-element">
      <span class="links-of-author-item">
        <a href="https://github.com/chenypic" title="GitHub &amp;rarr; https:&#x2F;&#x2F;github.com&#x2F;chenypic" rel="noopener" target="_blank"><i class="fa fa-fw fa-github"></i>GitHub</a>
      </span>
      <span class="links-of-author-item">
        <a href="/mailto:cherycheny@gmail.com" title="E-Mail &amp;rarr; mailto:cherycheny@gmail.com" rel="noopener" target="_blank"><i class="fa fa-fw fa-envelope"></i>E-Mail</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://weibo.com/2923640394" title="Weibo &amp;rarr; https:&#x2F;&#x2F;weibo.com&#x2F;2923640394" rel="noopener" target="_blank"><i class="fa fa-fw fa-weibo"></i>Weibo</a>
      </span>
      <span class="links-of-author-item">
        <a href="https://twitter.com/cherycheny" title="Twitter &amp;rarr; https:&#x2F;&#x2F;twitter.com&#x2F;cherycheny" rel="noopener" target="_blank"><i class="fa fa-fw fa-twitter"></i>Twitter</a>
      </span>
  </div>



      </div>

    </div>
  </aside>
  <div id="sidebar-dimmer"></div>


      </div>
    </main>

    <footer class="footer">
      <div class="footer-inner">
        

<div class="copyright">
  
  &copy; 
  <span itemprop="copyrightYear">2021</span>
  <span class="with-love">
    <i class="fa fa-user"></i>
  </span>
  <span class="author" itemprop="copyrightHolder">Wei Cheney</span>
</div>
  <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> 强力驱动 v4.0.0
  </div>
  <span class="post-meta-divider">|</span>
  <div class="theme-info">主题 – <a href="https://theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Gemini</a> v7.5.0
  </div>

        
<div class="busuanzi-count">
  <script async src="https://busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>
    <span class="post-meta-item" id="busuanzi_container_site_uv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-user"></i>
      </span>
      <span class="site-uv" title="总访客量">
        <span id="busuanzi_value_site_uv"></span>
      </span>
    </span>
    <span class="post-meta-divider">|</span>
    <span class="post-meta-item" id="busuanzi_container_site_pv" style="display: none;">
      <span class="post-meta-item-icon">
        <i class="fa fa-eye"></i>
      </span>
      <span class="site-pv" title="总访问量">
        <span id="busuanzi_value_site_pv"></span>
      </span>
    </span>
</div>












        
      </div>
    </footer>
  </div>

  
  <script src="/lib/anime.min.js"></script>
  <script src="/lib/velocity/velocity.min.js"></script>
  <script src="/lib/velocity/velocity.ui.min.js"></script>
<script src="/js/utils.js"></script><script src="/js/motion.js"></script>
<script src="/js/schemes/pisces.js"></script>
<script src="/js/next-boot.js"></script>



  
















  

  
      
<script type="text/x-mathjax-config">
    MathJax.Ajax.config.path['mhchem'] = '//cdn.jsdelivr.net/npm/mathjax-mhchem@3';

  MathJax.Hub.Config({
    tex2jax: {
      inlineMath: [ ['$', '$'], ['\\(', '\\)'] ],
      processEscapes: true,
      skipTags: ['script', 'noscript', 'style', 'textarea', 'pre', 'code']
    },
    TeX: {
        extensions: ['[mhchem]/mhchem.js'],
      equationNumbers: {
        autoNumber: 'AMS'
      }
    }
  });

  MathJax.Hub.Register.StartupHook('TeX Jax Ready', function() {
    MathJax.InputJax.TeX.prefilterHooks.Add(function(data) {
      if (data.display) {
        var next = data.script.nextSibling;
        while (next && next.nodeName.toLowerCase() === '#text') {
          next = next.nextSibling;
        }
        if (next && next.nodeName.toLowerCase() === 'br') {
          next.parentNode.removeChild(next);
        }
      }
    });
  });

  MathJax.Hub.Queue(function() {
    var all = MathJax.Hub.getAllJax(), i;
    for (i = 0; i < all.length; i += 1) {
      element = document.getElementById(all[i].inputID + '-Frame').parentNode;
      if (element.nodeName.toLowerCase() == 'li') {
        element = element.parentNode;
      }
      element.classList.add('has-jax');
    }
  });
</script>
<script>
  NexT.utils.getScript('//cdn.jsdelivr.net/npm/mathjax@2/MathJax.js?config=TeX-AMS-MML_HTMLorMML', () => {
    MathJax.Hub.Typeset();
  }, window.MathJax);
</script>

    

  

<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.css">

<script>
  NexT.utils.getScript('//cdn.jsdelivr.net/npm/gitalk@1/dist/gitalk.min.js', () => {
    var gitalk = new Gitalk({
      clientID: 'b518bd54c879c20706b4',
      clientSecret: 'f601dee7ed1d248ae971191226d140acbb9ca107',
      repo: 'PingLun',
      owner: 'chenypic',
      admin: ['chenypic'],
      id: '5f2219a832737935df51578c03753853',
        language: '',
      distractionFreeMode: 'true'
    });
    gitalk.render('gitalk-container');
  }, window.Gitalk);
</script>

</body>
</html>
